{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "from torch import nn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[ 0.0829,  0.1673,  0.1473, -0.3008, -0.0329],\n",
       "        [ 0.0419,  0.0951,  0.1939, -0.2574, -0.1442]],\n",
       "       grad_fn=<AddmmBackward0>)"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "net = nn.Sequential(nn.Linear(6, 32), nn.ReLU(), nn.Linear(32, 5))\n",
    "\n",
    "X = torch.rand(size=(2, 6))\n",
    "net(X)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 参数访问"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "OrderedDict([('weight', tensor([[ 0.1750, -0.1266, -0.0747,  0.1658, -0.0264, -0.1744,  0.1420, -0.0990,\n",
      "         -0.0957, -0.0262,  0.1496,  0.1587,  0.0783,  0.1257,  0.1188,  0.0879,\n",
      "         -0.1371,  0.0687, -0.0260,  0.0244,  0.0657, -0.0384,  0.0183,  0.1205,\n",
      "          0.1466,  0.0107,  0.0169, -0.0251,  0.0741, -0.1404, -0.1268,  0.0145],\n",
      "        [-0.1282, -0.0595, -0.1463,  0.1265,  0.1051, -0.1104,  0.0160, -0.1301,\n",
      "          0.0662,  0.0738,  0.0524,  0.0899,  0.0196,  0.1540, -0.1146,  0.1629,\n",
      "          0.1099, -0.0620, -0.0738, -0.0824,  0.1511, -0.1202, -0.1136, -0.1336,\n",
      "         -0.0296, -0.0456, -0.1512, -0.0202, -0.0101, -0.1291,  0.0989, -0.0457],\n",
      "        [-0.1699,  0.0702,  0.1681,  0.0787,  0.0750,  0.1059, -0.0201, -0.0261,\n",
      "          0.1026, -0.1049, -0.0378,  0.0345,  0.0168,  0.0319,  0.0230, -0.0757,\n",
      "         -0.0689, -0.1686, -0.0295,  0.1317,  0.0705, -0.0572,  0.0130,  0.1563,\n",
      "          0.0781,  0.0826, -0.1501, -0.0201,  0.0308, -0.1580,  0.0367, -0.1638],\n",
      "        [-0.0432, -0.0894,  0.0460, -0.0473,  0.0554, -0.1316, -0.0728, -0.0257,\n",
      "         -0.0544, -0.0652, -0.1362, -0.0153, -0.0469,  0.0284,  0.0266,  0.1741,\n",
      "          0.0592, -0.0740, -0.0647, -0.0881, -0.0769, -0.1442,  0.0850,  0.0438,\n",
      "          0.1709,  0.1374,  0.0917, -0.1316, -0.0991, -0.0247, -0.0701, -0.1335],\n",
      "        [-0.1347,  0.1458, -0.0527, -0.0927, -0.1613, -0.0335, -0.1021, -0.0258,\n",
      "         -0.1304,  0.1058, -0.1652, -0.1146,  0.0815, -0.1515,  0.0531, -0.1282,\n",
      "         -0.0242, -0.0145, -0.0492, -0.0454,  0.0820, -0.0151,  0.1763,  0.1478,\n",
      "          0.1161, -0.0065, -0.1709,  0.1488,  0.1669,  0.1628, -0.0947, -0.1766]])), ('bias', tensor([ 0.1524,  0.1005,  0.1051, -0.1545,  0.1302]))])\n"
     ]
    }
   ],
   "source": [
    "print(net[2].state_dict())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Parameter containing:\n",
      "tensor([ 0.1524,  0.1005,  0.1051, -0.1545,  0.1302], requires_grad=True)\n",
      "<class 'torch.nn.parameter.Parameter'>\n",
      "tensor([ 0.1524,  0.1005,  0.1051, -0.1545,  0.1302])\n"
     ]
    }
   ],
   "source": [
    "print(net[2].bias)\n",
    "print(type(net[2].bias))\n",
    "print(net[2].bias.data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('0.weight', torch.Size([32, 6])) ('0.bias', torch.Size([32])) ('2.weight', torch.Size([5, 32])) ('2.bias', torch.Size([5]))\n"
     ]
    }
   ],
   "source": [
    "print(*[(name, param.shape) for name, param in net.named_parameters()])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "*为可变参数，可实现解包和参数传递"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[-0.3244, -0.1870, -0.1605, -0.2690, -0.0781,  0.0659],\n",
       "        [-0.1154,  0.2753, -0.0730, -0.2066, -0.2200, -0.1055],\n",
       "        [-0.2581, -0.0661,  0.2867, -0.3433, -0.2017,  0.0949],\n",
       "        [-0.2062,  0.0956,  0.0757,  0.3952, -0.2008,  0.3672],\n",
       "        [ 0.3484, -0.2089,  0.3125, -0.1445, -0.0280, -0.0319],\n",
       "        [ 0.1522,  0.3083,  0.2906,  0.1095, -0.1064, -0.1783],\n",
       "        [ 0.0077,  0.3638, -0.0692,  0.2769,  0.0561, -0.3666],\n",
       "        [-0.2088,  0.3370, -0.1944, -0.3977, -0.2717, -0.1839],\n",
       "        [ 0.2573,  0.1539, -0.0145, -0.1699, -0.0174,  0.0726],\n",
       "        [-0.3419, -0.0346,  0.0906,  0.1605,  0.2368,  0.0423],\n",
       "        [-0.3451,  0.3344,  0.2080, -0.0440, -0.2952,  0.1897],\n",
       "        [-0.1617, -0.3107, -0.1400,  0.1852,  0.2839, -0.1291],\n",
       "        [ 0.3179, -0.3239,  0.1219,  0.0780, -0.2584,  0.1264],\n",
       "        [ 0.2665, -0.0473, -0.1491,  0.1719, -0.3326, -0.3748],\n",
       "        [-0.2203, -0.2609, -0.2336,  0.3417, -0.0303, -0.2888],\n",
       "        [-0.3237, -0.0495,  0.2442, -0.1660,  0.3849, -0.2541],\n",
       "        [ 0.2340,  0.1650,  0.1952,  0.0905,  0.0683,  0.0805],\n",
       "        [-0.1509, -0.2077,  0.0235,  0.3099,  0.3845, -0.1423],\n",
       "        [-0.3708,  0.0916, -0.2311,  0.0844,  0.2324,  0.1607],\n",
       "        [ 0.2908,  0.0655, -0.0008, -0.3000,  0.2325,  0.2411],\n",
       "        [ 0.1904, -0.3283,  0.1209,  0.3099,  0.1506,  0.3310],\n",
       "        [-0.3753, -0.0083, -0.0449, -0.1618,  0.3878, -0.1409],\n",
       "        [ 0.1156, -0.3930, -0.1556, -0.2880,  0.1960, -0.3567],\n",
       "        [ 0.1558, -0.2842, -0.0282,  0.3384, -0.2059,  0.1469],\n",
       "        [ 0.1231,  0.2954,  0.2737,  0.3000, -0.2944,  0.3925],\n",
       "        [ 0.1158,  0.3289,  0.0801, -0.1753, -0.0189, -0.3857],\n",
       "        [-0.0297,  0.3525,  0.2127, -0.0421, -0.3635, -0.0800],\n",
       "        [-0.3956,  0.0893, -0.2270,  0.0574, -0.1724, -0.0820],\n",
       "        [-0.3754,  0.1202, -0.1598,  0.1167, -0.3292,  0.2751],\n",
       "        [ 0.3274, -0.1557, -0.2233,  0.1468,  0.1797, -0.2951],\n",
       "        [ 0.2056,  0.1715,  0.1675, -0.2239, -0.1210, -0.0103],\n",
       "        [-0.3479, -0.4050,  0.1559, -0.3110,  0.2113,  0.0372]])"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "net.state_dict()['0.weight'].data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##  嵌套块参数访问"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[-0.1629],\n",
       "        [-0.1629]], grad_fn=<AddmmBackward0>)"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def block1():\n",
    "    return nn.Sequential(nn.Linear(6, 16), nn.ReLU(),\n",
    "                        nn.Linear(16, 6), nn.ReLU())\n",
    "\n",
    "def block2():\n",
    "    net = nn.Sequential()\n",
    "    for i in range(4):\n",
    "        net.add_module(f'block {i}', block1())\n",
    "    return net\n",
    "\n",
    "rgnet = nn.Sequential(block2(), nn.Linear(6, 1))\n",
    "rgnet(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Sequential(\n",
       "  (0): Sequential(\n",
       "    (block 0): Sequential(\n",
       "      (0): Linear(in_features=6, out_features=16, bias=True)\n",
       "      (1): ReLU()\n",
       "      (2): Linear(in_features=16, out_features=6, bias=True)\n",
       "      (3): ReLU()\n",
       "    )\n",
       "    (block 1): Sequential(\n",
       "      (0): Linear(in_features=6, out_features=16, bias=True)\n",
       "      (1): ReLU()\n",
       "      (2): Linear(in_features=16, out_features=6, bias=True)\n",
       "      (3): ReLU()\n",
       "    )\n",
       "    (block 2): Sequential(\n",
       "      (0): Linear(in_features=6, out_features=16, bias=True)\n",
       "      (1): ReLU()\n",
       "      (2): Linear(in_features=16, out_features=6, bias=True)\n",
       "      (3): ReLU()\n",
       "    )\n",
       "    (block 3): Sequential(\n",
       "      (0): Linear(in_features=6, out_features=16, bias=True)\n",
       "      (1): ReLU()\n",
       "      (2): Linear(in_features=16, out_features=6, bias=True)\n",
       "      (3): ReLU()\n",
       "    )\n",
       "  )\n",
       "  (1): Linear(in_features=6, out_features=1, bias=True)\n",
       ")"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rgnet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Parameter containing:\n",
       "tensor([-0.1475,  0.2333,  0.1465, -0.1582, -0.0535,  0.0894],\n",
       "       requires_grad=True)"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rgnet[0][1][2].bias"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 参数初始化"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(tensor([ 0.0254,  0.0100, -0.0078, -0.0052, -0.0062, -0.0055]),\n",
       " torch.Size([32]))"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def init_normal(m):\n",
    "    if type(m) == nn.Linear:\n",
    "        nn.init.normal_(m.weight, mean=0, std=0.01)\n",
    "        nn.init.zeros_(m.bias)\n",
    "\n",
    "net.apply(init_normal)\n",
    "net[0].weight.data[0], net[0].bias.data.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 初始化为常数项"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(tensor([2., 2., 2., 2., 2., 2.]), tensor(0.))"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def init_constant(m):\n",
    "    if type(m) == nn.Linear:\n",
    "        nn.init.constant_(m.weight, 2)\n",
    "        nn.init.zeros_(m.bias)\n",
    "        \n",
    "rgnet.apply(init_constant)\n",
    "rgnet[0][0][0].weight.data[0], rgnet[0][2][2].bias.data[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([ 0.1862, -0.3041, -0.0155, -0.2757,  0.3181,  0.3543])\n",
      "tensor([[42., 42., 42., 42., 42., 42., 42., 42., 42., 42., 42., 42., 42., 42.,\n",
      "         42., 42., 42., 42., 42., 42., 42., 42., 42., 42., 42., 42., 42., 42.,\n",
      "         42., 42., 42., 42.],\n",
      "        [42., 42., 42., 42., 42., 42., 42., 42., 42., 42., 42., 42., 42., 42.,\n",
      "         42., 42., 42., 42., 42., 42., 42., 42., 42., 42., 42., 42., 42., 42.,\n",
      "         42., 42., 42., 42.],\n",
      "        [42., 42., 42., 42., 42., 42., 42., 42., 42., 42., 42., 42., 42., 42.,\n",
      "         42., 42., 42., 42., 42., 42., 42., 42., 42., 42., 42., 42., 42., 42.,\n",
      "         42., 42., 42., 42.],\n",
      "        [42., 42., 42., 42., 42., 42., 42., 42., 42., 42., 42., 42., 42., 42.,\n",
      "         42., 42., 42., 42., 42., 42., 42., 42., 42., 42., 42., 42., 42., 42.,\n",
      "         42., 42., 42., 42.],\n",
      "        [42., 42., 42., 42., 42., 42., 42., 42., 42., 42., 42., 42., 42., 42.,\n",
      "         42., 42., 42., 42., 42., 42., 42., 42., 42., 42., 42., 42., 42., 42.,\n",
      "         42., 42., 42., 42.]])\n"
     ]
    }
   ],
   "source": [
    "def xavier(m):\n",
    "    if type(m) == nn.Linear:\n",
    "         nn.init.xavier_uniform_(m.weight)\n",
    "def init_42(m):\n",
    "    if type(m) == nn.Linear:\n",
    "        nn.init.constant_(m.weight, 42)\n",
    "\n",
    "net[0].apply(xavier)  # 第一层用xavier初始化\n",
    "net[2].apply(init_42)  # 第二层用常数42初始化\n",
    "print(net[0].weight.data[0])\n",
    "print(net[2].weight.data)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 自定义初始化"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Init weight torch.Size([32, 6])\n",
      "Init weight torch.Size([5, 32])\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "tensor([[-0.0000,  0.0000,  0.0000, -8.3725,  0.0000, -0.0000],\n",
       "        [ 0.0000,  0.0000,  8.5822, -0.0000,  9.9192, -7.0777]],\n",
       "       grad_fn=<SliceBackward0>)"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def my_init(m):\n",
    "    if type(m) == nn.Linear:\n",
    "        print(\"Init\", *[(name, param.shape)\n",
    "                        for name, param in m.named_parameters()][0])\n",
    "        nn.init.uniform_(m.weight, -10, 10)\n",
    "        m.weight.data *= m.weight.data.abs() >= 5\n",
    "\n",
    "net.apply(my_init)\n",
    "net[0].weight[:2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([42.0000,  2.0000,  2.0000, -6.3725,  2.0000,  2.0000])"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "net[0].weight.data[:] += 2\n",
    "net[0].weight.data[0][0] = 42\n",
    "net[0].weight.data[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
